{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Listing 3.1 – Importing necessary packages\n",
    "\n",
    "#Import required libraries\n",
    "import torch as tch\n",
    "import torch.nn as nn\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import make_blobs\n",
    "from matplotlib import pyplot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: torch.Size([4000, 64])\n",
      "x_test.shape: torch.Size([1000, 64])\n",
      "Y_train.shape: torch.Size([4000, 1])\n",
      "y_test.shape: torch.Size([1000, 1])\n",
      "X.dtype torch.float32\n",
      "y.dtype torch.float32\n"
     ]
    }
   ],
   "source": [
    "#Listing 3.2 – Creating a toy dataset\n",
    "\n",
    "samples = 5000\n",
    "\n",
    "#Let’s divide the toy dataset into training (80%) and rest for validation.\n",
    "train_split = int(samples*0.8)\n",
    "\n",
    "#Create a dummy classification dataset\n",
    "X, y = make_blobs(n_samples=samples, centers=2, n_features=64, cluster_std=10, random_state=2020)\n",
    "y = y.reshape(-1,1)\n",
    "\n",
    "#Convert the numpy datasets to Torch Tensors\n",
    "X,y = tch.from_numpy(X),tch.from_numpy(y)\n",
    "X,y =X.float(),y.float()\n",
    "\n",
    "#Split the datasets inot train and test(validation)\n",
    "X_train, x_test = X[:train_split], X[train_split:]\n",
    "Y_train, y_test = y[:train_split], y[train_split:]\n",
    "\n",
    "#Print shapes of each dataset\n",
    "print(\"X_train.shape:\",X_train.shape)\n",
    "print(\"x_test.shape:\",x_test.shape)\n",
    "print(\"Y_train.shape:\",Y_train.shape)\n",
    "print(\"y_test.shape:\",y_test.shape)\n",
    "print(\"X.dtype\",X.dtype)\n",
    "print(\"y.dtype\",y.dtype)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Listing 3.3 – Define a feed forward neural network\n",
    "\n",
    "\n",
    "#Define a neural network with 3 hidden layers and 1 output layer\n",
    "#Hidden Layers will have 64,256 and 1024 neurons\n",
    "#Output layers will have 1 neuron\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        tch.manual_seed(2020)\n",
    "        self.fc1 = nn.Linear(64, 256)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(256, 1024)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.out = nn.Linear(1024, 1)\n",
    "        self.final = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        op = self.fc1(x)\n",
    "        op = self.relu1(op)        \n",
    "        op = self.fc2(op)\n",
    "        op = self.relu2(op)\n",
    "        op = self.out(op)\n",
    "        y = self.final(op)\n",
    "        return y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 - Loss:106.7566\n",
      "Epoch: 2 - Loss:11.5689\n",
      "Epoch: 3 - Loss:7.8169\n",
      "Epoch: 4 - Loss:0.2327\n",
      "Epoch: 5 - Loss:0.0313\n",
      "Epoch: 6 - Loss:0.0034\n",
      "Epoch: 7 - Loss:0.0019\n",
      "Epoch: 8 - Loss:0.0012\n",
      "Epoch: 9 - Loss:0.0009\n",
      "Epoch: 10 - Loss:0.0007\n"
     ]
    }
   ],
   "source": [
    "#Listing 3.4 –Define loss, optimizer and Training function for  neural network\n",
    "\n",
    "#Define function for training a network\n",
    "def train_network(model,optimizer,loss_function \\\n",
    ",num_epochs,batch_size,X_train,Y_train):\n",
    "    #Explicitly start model training\n",
    "    model.train()\n",
    "\n",
    "    loss_across_epochs = []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss= 0.0\n",
    "\n",
    "\n",
    "        for i in range(0,X_train.shape[0],batch_size):\n",
    "\n",
    "            #Extract train batch from X and Y\n",
    "            input_data = X_train[i:min(X_train.shape[0],i+batch_size)]\n",
    "            labels = Y_train[i:min(X_train.shape[0],i+batch_size)]\n",
    "\n",
    "            #set the gradients to zero before starting to do backpropragation \n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            #Forward pass\n",
    "            output_data  = model(input_data)\n",
    "\n",
    "            #Caculate loss\n",
    "            loss = loss_function(output_data, labels)\n",
    "\n",
    "            #Backpropogate\n",
    "            loss.backward()\n",
    "\n",
    "            #Update weights\n",
    "            optimizer.step()\n",
    "\n",
    "            train_loss += loss.item() * batch_size\n",
    "\n",
    "        print(\"Epoch: {} - Loss:{:.4f}\".format(epoch+1,train_loss ))\n",
    "        loss_across_epochs.extend([train_loss])        \n",
    "\n",
    "    #Predict\n",
    "    y_test_pred = model(x_test)\n",
    "    a =np.where(y_test_pred>0.5,1,0)\n",
    "    return(loss_across_epochs)\n",
    "###------------END OF FUNCTION--------------\n",
    "\n",
    "#Create an object of the Neural Network class\n",
    "model = NeuralNetwork()\n",
    "\n",
    "#Define loss function\n",
    "loss_function = nn.BCELoss()  #Binary Crosss Entropy Loss\n",
    "\n",
    "#Define Optimizer\n",
    "adam_optimizer = tch.optim.Adam(model.parameters(),lr= 0.001)\n",
    "\n",
    "#Define epochs and batch size\n",
    "num_epochs = 10\n",
    "batch_size=16\n",
    "\n",
    "\n",
    "#Calling the function for training and pass model, optimizer, loss and related paramters\n",
    "adam_loss = train_network(model,adam_optimizer \\\n",
    ",loss_function,num_epochs,batch_size,X_train,Y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSProp...\n",
      "Epoch: 1 - Loss:5794.6734\n",
      "Epoch: 2 - Loss:1680.3092\n",
      "Epoch: 3 - Loss:1169.5457\n",
      "Epoch: 4 - Loss:1518.7088\n",
      "Epoch: 5 - Loss:1727.5753\n",
      "Epoch: 6 - Loss:661.7122\n",
      "Epoch: 7 - Loss:532.6023\n",
      "Epoch: 8 - Loss:2613.1597\n",
      "Epoch: 9 - Loss:283.5713\n",
      "Epoch: 10 - Loss:1058.1581\n",
      "Adam...\n",
      "Epoch: 1 - Loss:106.7566\n",
      "Epoch: 2 - Loss:11.5689\n",
      "Epoch: 3 - Loss:7.8169\n",
      "Epoch: 4 - Loss:0.2327\n",
      "Epoch: 5 - Loss:0.0313\n",
      "Epoch: 6 - Loss:0.0034\n",
      "Epoch: 7 - Loss:0.0019\n",
      "Epoch: 8 - Loss:0.0012\n",
      "Epoch: 9 - Loss:0.0009\n",
      "Epoch: 10 - Loss:0.0007\n",
      "SGD...\n",
      "Epoch: 1 - Loss:801.0526\n",
      "Epoch: 2 - Loss:131.7263\n",
      "Epoch: 3 - Loss:296.2784\n",
      "Epoch: 4 - Loss:240.0572\n",
      "Epoch: 5 - Loss:248.2811\n",
      "Epoch: 6 - Loss:248.2784\n",
      "Epoch: 7 - Loss:248.2759\n",
      "Epoch: 8 - Loss:248.2733\n",
      "Epoch: 9 - Loss:248.2708\n",
      "Epoch: 10 - Loss:248.2684\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd8VfX5wPHPkx0gCWQQNoS9AiE3grgKMgUVLFWGAxFFrWirVuWn1lmttFXrVoQKakVUqCCCBWUodSBB9t6ElUFICCEh4/v745zEG8i4GTc343m/XveVe/ZzTpL73O843yPGGJRSSilXeXk6AKWUUrWLJg6llFLloolDKaVUuWjiUEopVS6aOJRSSpWLJg6llFLloolDKUBEVonI7VWwny4iskFETovIfVUR23n7HyAiCU7TW0VkgP1eROQ9EUkVkbX2vLtF5ISIZIhIWFXHU5OIyI0isqyC214uIjurOqa6ysfTAdQ3InIAuN0Y87WnY1Fu8TCw0hgTUx0HM8b0cJq8DBgCtDLGnBERX+Al4GJjzMbqiMeZiMwGEowxj7th3+2A/YCvMSYXwBjzb+DfFdmfMeY7oEtVxVfXaYlDlYuI6JeN0rUFtlZkwyq4tm2BA8aYM/Z0JBBQiXi8KxlPvVan/1eMMfqqxhdwABhcwrI7gD3ASWAR0MKeL8DLQCKQDmwGetrLRgDbgNPAEeBPJey7A7ACSAGSsb6ZNXZa3hpYACTZ67xuz78V+J99/BTgL1hfOB4HDtoxvQ+E2OsHAB/a654CfgYinfa1z451P3BjCbF6AdOAvfZ+PgFC7WXtAANMAY4Cx5zPGfAH/mkvO2q/93daPgrYYF/HvcBwe/4q4Fn7XE8Dy4Dwss7pvLhXAHlAFpABdAZC7OuTZF+vxwGvkq5tMfsMBGYDqfbv+SGsb/FF/p6AyfZx8+xjzwXO2NcqA1hhr98VWI71N7YTuMFpX7OBt4Al9raD7ev5D+AQcAJ4Gwi01x8AJAAP2n8Hx4BJ9rIpQA5wzj7+FyX8ri+xr2ea/fMSp2WrgL8Ca+3f10Knv4NDTueWAfS3r+cap+0N8Htgt/07fRbr/+B7e3+fAH7O52K/H+u03wwgG1jl9PdV1vV4BDgOfACEA4ux/m5OAt8V/P5r88vjAdS3FyUkDuBKrA/0WPuP8zXgW3vZMCAeaIyVRLoBze1lx4DL7fdNgNgSjtsRqxrDH4gAvgX+aS/zBjZifYA1xPqgvMxediuQC9yLVbUZCNyGleDaA42wEs4H9vp3Al8ADez9OoBge7/pQBd7veZAjxJi/QPwI9DKjvcdYK69rJ39gTDX3mc01ofyYHv5M/a2Te3z/B541l7WF+sDaghWcmoJdLWXrcJKJJ3tc1wFvFDaOZUQ+yqsqsiC6fexPvCC7Nh3AZNLurbF7O8FrA+bUKzkvoViEofT/pw/OAuulY893RA4DEyyj9cH62+uu718tn19LrWvTwDW38Qi+/hB9nX4q73+ADv+ZwBfrC8xmUATp/1dkAyd4gvFSog32/GMt6fDnK7lEaCnHft84MPizq2E8zf2tQ8GemAlgG+w/m5DsBLxRKdzSSgmxmBgO3CnPe3K9ZiO9XcbiJX43ravjy9wOSCe/hyq9OeYpwOoby9KThyzgL85TTfC+sbWDiup7AIu5rxvK1jffO6khA+yUuIYDfxiv++P9eHrU8x6twKHzpv3DfB7p+kudqw+WEnle6DXeds0xPrWNYZiPiDPW3c7MMhpurnT/gs+MLo6Lf8bMMt+vxcY4bRsGFb1DVgJ6OUSjrkKeNxp+vfAV/b7Ys+plP3cbr/3xvrG3d1p+Z38+u31gmtbzP72YZeK7OkpVDxxjAW+O2//7wBP2u9nA+87LROskkcHp3n9gf32+wHAWYp+eCditakU7K+0xHEzsPa8eT8AtzpdyxeclnW3r6f3+edWwvkb4FKn6XjgEafpF/n1y9MAzkscWMlzMfBWOa7HOSDAafkzWMmrY3n+P2v6S9s4ao4WWFUZABhjMrCqL1oaY1YArwNvAIkiMkNEgu1Vx2B90zsoIqtFpH9xOxeRSBH5WESOiEg6VtVLuL24NXDQ2I2MxThcWqz2ex+sOvUPgP8CH4vIURH5m4j4GqvefSxwF3BMRL4Uka4lHK8t8B8ROSUip7ASSZ69/+JiOmjHVFJsBctaYyWWkhx3ep+Jlbwp6ZxK2U+BcKxvmefH07KE8yhOCy4814pqC/QruK72tb0RaFZCPBFYpax4p/W/sucXSDnv78b5upXl/N8VlH59DmJdz3Bcd8Lp/dlipkuL9TmsUkVB7zhXrkeSMSbLafrvWKXzZSKyT0SmlSP2GksTR81xFOsfGwARaQiEYRXVMca8aoxxYH3r6oxV140x5mdjzCisqpnPsepti/M81jewaGNMMHAT1jcosP4525TSmGdKixVog1VEP2GMyTHGPG2M6Y5Vf301cIsd63+NMUOwShA7gHdLON5h4CpjTGOnV4Ax5ojTOq3PO/7RUmIrWHYYq467XEo7pzIkY5WUzo/H+TzOv7bnO8aF51pRh4HV513XRsaYu0uIJxnrw7WH0/ohxhhXE0NZ53b+7wouvD7nn3uOHVdZ+64UERmHVXX2O2NMjj3bletRJC5jzGljzIPGmPbAtcADIjLInbFXB00cnuErIgFOLx+sOvtJIhIjIv5YH/Q/GWMOiMhFItLP/pZ7BqsRNF9E/Oy+6yH2H3c6kF/CMYOwGvrSRKQlduKxrcX6gHpBRBraMV1aSvxzgftFJEpEGtmxzjPG5IrIQBGJtnvkpGP9o+fbJZ5RdkLMtmMpKda3gedEpC2AiESIyKjz1vmziDQQkR5YdfbznGJ73N4mHHgCq3QFVnXgJBEZJCJeItKylFJPoZLOqaztjDF5WIn8OREJss/nAad4XPEJ8H8i0kREWmG1h1TUYqCziNwsIr726yIR6VZC/PlYyf1lEWkKYF+zYS4e7wRWe0JJltjxTBARHxEZi/XFaLHTOjeJSHcRaYBV7fOZfV2TsH4Hpe2/QkSkD1Yb42hjTFLB/IpcDxG5WkQ6iohgtR/l4cLfTk2nicMzlmB9cyl4PWWs+zr+jNUAeAzrm/E4e/1grD/YVKziegpWERiseuIDdvXTXVhVD8V5GqvhPQ34EqtBGyj8gLsGqwH9EFbPkLGlxP8vrOqbb7F6R2Xx6wdaM+AzrA/Y7cBqe10vrA/No1i9S34D3E3xXsFqgFwmIqexGrv7nbfOaqwqgG+AfxhjCm78+guwDtiE1ftsvT0PY8xarCTzsn0dVnPhN97ilHROrrgXK9nvA9YAH2FdP1c9jfU734/V08vV417AGHMaGIr1d3UUq2quoCG3JI9gXecf7b+xr3H9fodZQHe7WufzYuJJwSq9PYj1N/0wcLUxJtlptQ+w2kqOYzXW32dvm4lVlfQ/e/8XuxiTK0ZhdTRZY984mSEiS+1l5b0enex1MrDab940xqyswlg9QuwGHKVqheJu/FJ1k4iswupFNdPTsaiitMShlFKqXDRxKKWUKhetqlJKKVUuWuJQSilVLm4dhEtEGgMzsYYMMFh34O7E6jrZDuuu1xuMMal2d7VX+HXYgluNMevt/UzEGuMHrDtR55R23PDwcNOuXbuqPh2llKrT4uPjk40xEWWt59aqKhGZgzXEwUwR8cO66/JR4KQx5gX7LsomxphHRGQEVtfFEVhdL18xxvQTkVCs7pVxWMknHnAYY1JLOm5cXJxZt26d285LKaXqIhGJN8bElbWe26qqRCQEuAKrLzfGmHPGmFNYfaQLSgxzsMZMwp7/vrH8CDQWkeZYYw0tN8actJPFcmC4u+JWSilVOne2cURh3d35noj8IiIz7buGI40xx+x1jvPr+EMtKTouTYI9r6T5RYjIFBFZJyLrkpKSzl+slFKqirgzcfhg3an8ljGmD9bds0UG+DJWPVmV1JUZY2YYY+KMMXEREWVW0SmllKogdzaOJ2ANU/yTPf0ZVuI4ISLNjTHH7KqoRHv5EYoOaNbKnncEa7hi5/mr3Bi3UqoWyMnJISEhgaysrLJXVkUEBATQqlUrfH1dGeT5Qm5LHMaY4yJyWES6GGN2AoOwHpyyDZiI9YCaiVhj1YM1NtFUEfkYq3E8zU4u/wWeF5Em9npDgf9zV9xKqdohISGBoKAg2rVrh9UpU7nCGENKSgoJCQlERUVVaB/ufibuvcC/7R5V+7AGmPMCPhGRyViDt91gr7sEq0fVHqzuuJMAjDEnReRZrMdKAjxjjDnp5riVUjVcVlaWJo0KEBHCwsKoTFuwWxOHMWYDVjfa810wHr3d3nFPCfv5F+UbUVQpVQ9o0qiYyl43vXPcybGMY7y6/lWOZRwre2WllKqnNHE4ycjJ4N3N7/LT8Z/KXlkppYDPP/8cEWHHjh0AHDhwgMDAQPr06UO3bt3o27cvs2fPvmC70aNHc/HFRR8j8tRTTyEi7Nmzp3DeP//5T0SEmnRTsyYOJx0adyDEP4T4E/GeDkUpVUvMnTuXyy67jLlz5xbO69ChA7/88gvbt2/n448/5p///Cfvvfde4fJTp04RHx9PWloa+/btK7K/6OhoPv7448LpTz/9lB49erj/RMpBE4cTL/EitmmsJg6llEsyMjJYs2YNs2bNKvJh76x9+/a89NJLvPrqq4XzFixYwDXXXMO4ceMu2G706NEsXGh1Nt27dy8hISGEh4e77yQqwN29qmodR6SDlYdXkpiZSNMGTT0djlLKBU9/sZVtR9OrdJ/dWwTz5DWlf9NfuHAhw4cPp3PnzoSFhREfH09YWNgF68XGxhZWZYFVSnniiSeIjIxkzJgxPProo4XLgoODad26NVu2bGHhwoWMHTu2SGmlJtASx3kckQ4A1p9Y7+FIlFI13dy5cxk3bhwA48aNK1Jd5cx5MNkTJ06we/duLrvsMjp37oyvry9btmwpsn5BSeTzzz/nuuuuc98JVJCWOM7TNbQrgT6BrDuxjuFROpaiUrVBWSUDdzh58iQrVqxg8+bNiAh5eXmICPfcc+FdBb/88gvdunUD4JNPPiE1NbXw5rv09HTmzp3Lc889V7j+1VdfzUMPPURcXBzBwcHVc0LloCWO8/h4+RATEcP6RC1xKKVK9tlnn3HzzTdz8OBBDhw4wOHDh4mKiuLw4cNF1jtw4AB/+tOfuPfeewGrlPLVV19x4MABDhw4QHx8/AXtHA0aNGD69Ok89thj1XY+5aEljmI4Ih28vuF10rLTCPEP8XQ4SqkaaO7cuTzyyCNF5o0ZM4a//vWv7N27lz59+pCVlUVQUBD33Xcft956KwcOHODgwYNFuuFGRUUREhLCTz8VvQ2goAqsJqqTzxyv7IOc1h1fx6T/TuLVga8ysM3AKoxMKVVVtm/fXlj9o8qvuOvn8Qc51WbREdH4evlqdZVSShVDE0cx/L39iQ6P1vs5lFKqGJo4SuCIdLAtZRuZOZmeDkUppWoUTRwliI2MJc/ksTFpo6dDUUqpGkUTRwliImLwEi+trlJKqfNo4ihBI79GdA3tqolDKaXOo4mjFI5IB5uSNnEu75ynQ1FK1VDPPfccPXr0oFevXsTExPDTTz+Rm5vLo48+SqdOnYiJiSEmJqbIneHe3t7ExMTQo0cPevfuzYsvvkh+fr4Hz6J89AbAUjiaOvhg2wdsTdlKn6Z9PB2OUqqG+eGHH1i8eDHr16/H39+f5ORkzp07x+OPP87x48fZvHkzAQEBnD59mhdffLFwu8DAQDZs2ABAYmIiEyZMID09naefftpTp1IuWuIoRZ9IK1lodZVSqjjHjh0jPDwcf39/AMLDw2ncuDHvvvsur732GgEBAQAEBQXx1FNPFbuPpk2bMmPGDF5//XVqyw3ZWuIoRWhAKB1COrDuxDpuj77d0+EopUqydBoc31y1+2wWDVe9UOoqQ4cO5ZlnnqFz584MHjyYsWPH0qRJE9q0aUNQUJDLh2rfvj15eXkkJiYSGRlZ2cjdTkscZYiNjGVD4gby8vM8HYpSqoZp1KgR8fHxzJgxg4iICMaOHcuqVauKrPPee+8RExND69atLxgAsbbSEkcZHJEOPt31KTtTd9I9rLunw1FKFaeMkoE7eXt7M2DAAAYMGEB0dDTvvPMOhw4d4vTp0wQFBTFp0iQmTZpEz549ycsr/gvovn378Pb2pmnT2vHwOC1xlKHgwU7azqGUOt/OnTvZvXt34fSGDRvo0qULkydPZurUqWRlZQGQl5fHuXPF985MSkrirrvuYurUqYhItcRdWVriKEOzhs1o2agl60+s5+buN3s6HKVUDZKRkcG9997LqVOn8PHxoWPHjsyYMYOQkBD+/Oc/07NnT4KCgggMDGTixIm0aNECgLNnzxITE0NOTg4+Pj7cfPPNPPDAAx4+G9dp4nCBI9LBdwnfYYypNd8IlFLu53A4+P7774td9sILL/DCC8VXoZVUZVVbuLWqSkQOiMhmEdkgIuvseaEislxEdts/m9jzRUReFZE9IrJJRGKd9jPRXn+3iEx0Z8zFcUQ6SM1OZX/a/uo+tFJK1TjV0cYx0BgT4/RwkGnAN8aYTsA39jTAVUAn+zUFeAusRAM8CfQD+gJPFiSb6hLb1Mph8YnazqGUUp5oHB8FzLHfzwFGO81/31h+BBqLSHNgGLDcGHPSGJMKLAeGV2fAbYPbEhYQpg3kSimF+xOHAZaJSLyITLHnRRpjjtnvjwMFd7u0BJw7OSfY80qaX4SITBGRdSKyLikpqSrPARHBEenQxKGUUrg/cVxmjInFqoa6R0SucF5orPvrq+Qee2PMDGNMnDEmLiIioip2WURsZCzHzxznaMbRKt+3UkrVJm5NHMaYI/bPROA/WG0UJ+wqKOyfifbqR4DWTpu3sueVNL9axUVaTTRa6lBK1XduSxwi0lBEggreA0OBLcAioKBn1ERgof1+EXCL3bvqYiDNrtL6LzBURJrYjeJD7XnVqmPjjgT5BWniUEoVKhgevWfPnlxzzTWcOnUKgAMHDiAiPP7444XrJicn4+vry9SpUwHr5sEBAwYQExNDt27dmDLFqs1ftWoVISEhhfNr4oi57ixxRAJrRGQjsBb40hjzFfACMEREdgOD7WmAJcA+YA/wLvB7AGPMSeBZ4Gf79Yw9r1p5e3kT2zRWE4dSqlDB8OhbtmwhNDSUN954o3BZVFQUX375ZeH0p59+So8ePQqn77vvPu6//342bNjA9u3buffeewuXXX755WzYsIF169bx4Ycfsn79+iLHzc3NdeNZlc1ticMYs88Y09t+9TDGPGfPTzHGDDLGdDLGDC5IAnZvqnuMMR2MMdHGmHVO+/qXMaaj/XrPXTGXJTYylgPpB0g+m+ypEJRSNVT//v05cuTXWvQGDRrQrVs31q2zPsrmzZvHDTfcULj82LFjtGrVqnA6Ojr6gn02bNgQh8PBnj17mD17Ntdeey1XXnklgwYNwhjDQw89RM+ePYmOjmbevHmAVWK54oorGDlyJF26dOGuu+6q8odE6Z3j5VAwbtX6E+sZ2m6oh6NRShWYvnY6O07uqNJ9dg3tyiN9H3Fp3by8PL755hsmT55cZP64ceP4+OOPiYyMxNvbmxYtWnD0qNXB5v777+fKK6/kkksuYejQoUyaNInGjRsX2T4lJYUff/yRP//5z/z888+sX7+eTZs2ERoayvz589mwYQMbN24kOTmZiy66iCuusPofrV27lm3bttG2bVuGDx/OggUL+N3vflcFV8WigxyWQ/fQ7gT6BGp1lVIK+HXMqWbNmnHixAmGDBlSZPnw4cNZvnw5H3/8MWPHji2ybNKkSWzfvp3rr7+eVatWcfHFF5OdnQ3Ad999R58+fRg6dCjTpk0rrOIaMmQIoaGhAKxZs4bx48fj7e1NZGQkv/nNb/j5558B6Nu3L+3bt8fb25vx48ezZs2aKj1vLXGUg6+3L73Ce7E+cX3ZKyulqo2rJYOqVtDGkZmZybBhw3jjjTe47777Cpf7+fnhcDh48cUX2bZtG4sWLSqyfYsWLbjtttu47bbb6NmzJ1u2bAGsNo7FixdfcLyGDRu6FNf5Y+pV9Rh7WuIoJ0ekg50nd5J+Lt3ToSilaogGDRrw6quv8uKLL17QcP3ggw8yffr0wpJCga+++oqcnBwAjh8/TkpKCi1bXnBvc4kuv/xy5s2bR15eHklJSXz77bf07dsXsKqq9u/fT35+PvPmzeOyyy6r5BkWpYmjnByRDgyGDYkbPB2KUqoG6dOnD7169WLu3LlF5vfo0YOJEy8cm3XZsmX07NmT3r17M2zYMP7+97/TrFkzl4933XXX0atXL3r37s2VV17J3/72t8LtL7roIqZOnUq3bt2Iioriuuuuq9zJnUdqy8PRyyMuLs4U9GSoamdzz3LJ3Eu4pfst3O+43y3HUEqVbfv27XTr1s3TYdQ4q1at4h//+EexVV3Oirt+IhLvNCBtibTEUU6BPoH0COuhDeRKqXpLE0cFOCIdbE3eytncs54ORSmlihgwYECZpY3K0sRRAY5IB7kml81Jmz0dilL1Wl2saq8Olb1umjgqIKZpDIJodZVSHhQQEEBKSoomj3IyxpCSkkJAQECF96H3cVRAsF8wXUK7aOJQyoNatWpFQkICVf38nfogICCgyHAn5aWJo4Jim8ayYPcCcvJy8PX29XQ4StU7vr6+REVFeTqMekmrqirIEekgKy+LbSe3eToUpZSqVpo4Kig2MhbQBzsppeofTRwVFB4YTrvgdqw/oeNWKaXqF00cleCIdLD+xHry8vM8HYpSSlUbTRyV4Ih0cDrnNHtO7fF0KEopVW00cVRCwYOdtJ1DKVWfaOKohBaNWtCsYTNNHEqpekUTRyU5Ih3En4jXu1eVUvWGJo5KckQ6SMlK4WD6QU+HopRS1UITRyU5mlrtHPo4WaVUfaGJo5KiQqJo4t9E2zmUUvWGJo5KEpHCdg6llKoPNHFUgdjIWI5kHOH4meOeDkUppdzO7YlDRLxF5BcRWWxPR4nITyKyR0TmiYifPd/fnt5jL2/ntI//s+fvFJFh7o65vPR+DqVUfVIdJY4/ANudpqcDLxtjOgKpwGR7/mQg1Z7/sr0eItIdGAf0AIYDb4qIdzXE7bIuTbrQ0LehJg6lVL3g1sQhIq2AkcBMe1qAK4HP7FXmAKPt96Psaezlg+z1RwEfG2OyjTH7gT1AX3fGXV7eXt7ENI3RAQ+VUvWCu0sc/wQeBvLt6TDglDEm155OAFra71sChwHs5Wn2+oXzi9mmkIhMEZF1IrLOE08Ei4uMY2/aXk5mnaz2YyulVHVyW+IQkauBRGNMtdTfGGNmGGPijDFxERER1XHIIgraOX458Uu1H1sppaqTO0sclwLXisgB4GOsKqpXgMYiUvDI2lbAEfv9EaA1gL08BEhxnl/MNjVGj7Ae+Hv7E5+o7RxKqbrNbYnDGPN/xphWxph2WI3bK4wxNwIrgd/Zq00EFtrvF9nT2MtXGGsAqEXAOLvXVRTQCVjrrrgrys/bj+jwaG0gV0rVeZ64j+MR4AER2YPVhjHLnj8LCLPnPwBMAzDGbAU+AbYBXwH3GGNq5JOTHJEOdpzcQca5DE+HopRSblMticMYs8oYc7X9fp8xpq8xpqMx5npjTLY9P8ue7mgv3+e0/XPGmA7GmC7GmKXVEXNFOCId5Jt8NiZt9HQoSinlNnrneBXqHdEbb/HW6iqlVJ2miaMKNfBtQPew7po4lFJ1miaOKuaIdLA5eTPZedmeDkUppdyizMQhIg1FxMt+31lErhURX/eHVjvFNo0lJz+HzUmbPR2KUkq5hSsljm+BABFpCSwDbgZmuzOo2iw2MhbQAQ+VUnWXK4lDjDGZwG+BN40x12MNOKiKEeIfQqcmnTRxKKXqLJcSh4j0B24EvrTn1ajRaWua2KaxbEjaQG5+btkrK6VULeNK4vgj8H/Af4wxW0WkPdbd36oEcZFxnM09y46TOzwdilJKVTmfslYwxqwGVgPYjeTJxpj73B1YbebcztEzvKeHo1FKqarlSq+qj0QkWEQaAluAbSLykPtDq72aNmhK66DW2s6hlKqTXKmq6m6MScd64NJSIAqrZ5UqhSPSwfrE9eSb/LJXVkqpWsSVxOFr37cxGlhkjMkBjHvDqv0ckQ7SstPYe2qvp0NRSqkq5UrieAc4ADQEvhWRtkC6O4OqCwoe7KSPk1VK1TVlJg5jzKvGmJbGmBHGchAYWA2x1WqtGrWiaWBTbedQStU5rjSOh4jISwXP8xaRF7FKH6oUIoIj0kH8iXis51EppVTd4EpV1b+A08AN9isdeM+dQdUVjkgHiWcTSchI8HQoSilVZcq8jwPoYIwZ4zT9tIhscFdAdYnz/Rytg1qXsbZSStUOrpQ4zorIZQUTInIpcNZ9IdUdHRp3IMQ/RNs5lFJ1iisljruA90UkxJ5OBSa6L6S6w0u8iG0aqz2rlFJ1iiu9qjYaY3oDvYBexpg+QCe3R1ZHOCIdHDp9iMTMRE+HopRSVcLlJwAaY9LtO8gBXnZTPHWO3s+hlKprKvroWKnSKOqwrqFdCfQJZN2JdZ4ORSmlqkRFE4femOAiHy8fYiJiWJ+oJQ6lVN1QYuO4iGym+AQhQKTbIqqDHJEOXt/wOmnZaYT4h5S9gVJK1WCl9aq6utqiqOOc2zkGttHRWpRStVuJVVXGmIOlvcrasYgEiMhaEdkoIltF5Gl7fpSI/CQie0Rknoj42fP97ek99vJ2Tvv6P3v+ThEZVvnTrl7REdH4evlqdZVSqk6oaBuHK7KBK+2uvDHAcBG5GJgOvGyM6Yh1T8hke/3JQKo9/2V7PUSkOzAO6AEMB94UkVr1zHN/b3+iw6P1RkClVJ3gtsRhj6SbYU/62i8DXAl8Zs+fg/WcD4BR9jT28kEiIvb8j40x2caY/cAeoK+74nYXR6SDbSnbyMzJ9HQoSilVKe4scSAi3va4VonAcmAvcMoYk2uvkgC0tN+3BA4D2MvTgDDn+cVsU2s4Ih3kmTw2Jm30dChKKVUpFe1VZYwxvcrauTEmD4gRkcbAf4CuFQ20LCIyBZgC0KZNG3dWcgUwAAAgAElEQVQdpsJ6R/TGS7yIPxFP/xb9PR2OUkpVWLX0qjLGnBKRlUB/oLGI+NililbAEXu1I0BrIEFEfIAQIMVpfgHnbZyPMQOYARAXF1fj7jNp5NeIrqFdtZ1DKVXrubNXVYRd0kBEAoEhwHZgJfA7e7WJwEL7/SJ+HTzxd8AKYz0BaREwzu51FYU1Ttba8p+q5zkiHWxO3sy5vHOeDkUppSqsxMQhIqdFJL2Y12kRceWZ482BlSKyCfgZWG6MWQw8AjwgInuw2jBm2evPAsLs+Q8A0wCMMVuBT4BtwFfAPXYVWK3jaOogOy+brSlbPR2KUkpVWIlVVcaYoMrs2BizCehTzPx9FNMryhiTBVxfwr6eA56rTDw1QZ9I63LEn4inT9MLLo1SStUKLveqEpGmItKm4OXOoOqq0IBQOoR00HYOpVStVmbiEJFrRWQ3sB9YDRwAlro5rjorNjKWXxJ/IS+/Vta2KaWUSyWOZ4GLgV3GmChgEPCjW6OqwxyRDs7knGFn6k5Ph6KUUhXiSuLIMcakAF4i4mWMWQnEuTmuOqtgwEOtrlJK1VauJI5TItII+Bb4t4i8Apxxb1h1V7OGzWjZqKU+EVApVWu5kjhGAZnA/VjdYfcC17gzqLrOEekg/kQ81m0qSilVu5SaOOxRaBcbY/KNMbnGmDnGmFftqitVQY5IB6nZqexP2+/pUJRSqtxKTRz2jXb5IqKPratCsU1jAYhP1HYOpSpj4Z6FXLfwOs7mnvV0KPWKK1VVGcBmEZklIq8WvNwdWF3WNrgtYQFh2kCuVCXk5OXw2i+vsefUHr7c96Wnw6lXShvksMAC+6WqiIgUtnMopSpm8b7FnMg8QbBfMB/t+IgxncZgPcJHuVuZJQ5jzByssaJ+tNs45tjzVCU4Ih0cP3OcoxlHPR2KUrVOXn4es7bMontYdx5wPMDu1N2sO7HO02HVG67cOX4NsAGrRxUiEiMii9wdWF2n93MoVXHLDy3nYPpB7oi+gxHtRxDsF8zcHXM9HVa94Uobx1NYgxKeAjDGbADauzGmeqFj444E+QVp4lCqnIwxzNw0k6iQKK5scyWBPoGM6TSGFYdWcPzMcU+HVy+4eud42nnz8t0RTH3i7eVNbNNYTRxKldN3R75jZ+pOJvecjJdYH2Fju47FYJi3c56Ho6sfXEkcW0VkAuAtIp1E5DXgezfHVS/ERsZyIP0AyWeTPR2KUrWCMYZ3N71L84bNGdF+ROH8lo1a8ptWv2H+rvlk52V7MML6wZXEcS/QA8gGPgLSgD+6M6j6oqCdQ4cfUco18Sfi2ZC0gUk9J+Hr5Vtk2YRuE0jNTmXpfh28291cSRxdjTGPGWMusl+P2w9dUpXUPbQ7gT6BrE/UxKGUK2ZunkloQCjXdbzugmX9mvWjQ0gHPtr+kQ7n42auJI4XRWS7iDwrIj3dHlE94uvtS6/wXtrOoZQLtqZs5X9H/8ct3W8hwCfgguUiwviu49l+cjsbkzZ6IML6w5X7OAYCA4Ek4B0R2Swij7s9snrCEelg58mdpJ9z5THuStVfszbPIsg3iLFdxpa4zjUdriHIN4iPdnxUjZHVPy49OtYYc9wY8ypwF9Y9HU+4Nap6xBHpwGDYkLjB06EoVWPtO7WPrw9+zfhu42nk16jE9Rr4NmBUx1EsP7CcpMykaoywfnHlBsBuIvKUiGwBCnpUtXJ7ZPVEdEQ0Pl4+Wl2lVClmbZlFgE8AN3W7qcx1x3cdT57J49Ndn1ZDZPWTKyWOfwGpwFBjzABjzFvGmEQ3x1VvBPoE0iOshyYOpUpwJOMIX+77kjGdxtAkoEmZ67cJbsNlLS/j012fkpOXUw0R1j+uJI4rgW+AUBG5sEVKVZoj0sHW5K06NLRSxZi9ZTYiwsQeE13eZkK3CSSfTWbZwWVujKz+KjFxiIiPiPwNOATMAd4HDovI30TEt6TtVPk5Ih3kmlw2J232dChK1SjJZ5NZsHsBozqMolnDZi5vd0mLS2gb3FYbyd2ktBLH34FQoL0xxmGMiQU6AI2Bf1RHcPVFTNMYBNHqKqXO88G2D8g1uUzqOalc23mJF+O7jmdT0ia2Jm91U3T1V2mJ42rgDmPM6YIZxph04G5gRIlbqXIL9gumS2gXTRxKOUnLTmPeznkMazuMtsFty739qA6jaODTQEsdblBa4jCmmNsv7cfJlnlbpoi0FpGVIrJNRLaKyB/s+aEislxEdts/m9jzxX664B4R2SQisU77mmivv1tEXK/orEUckQ42Jm3UxjylbB/v+JgzOWeYHD25Qts38mvENR2uYen+paScTani6Oq30hLHNhG55fyZInITsMOFfecCDxpjugMXA/eISHdgGvCNMaYTVqP7NHv9q4BO9msK8JZ9vFDgSaAf1vDuTxYkm7oktmksWXlZbDu5zdOhKOVxmTmZfLj9Q37T6jd0Ce1S4f1M6DqBnPwc5u+eX4XRqdISxz1YH/arRORF+7UauA+ruqpUxphjxpj19vvTwHagJTAKq7Ed++do+/0o4H1j+RFoLCLNgWHAcmPMSWNMKrAcGF7uM63hYiOtApZWVykF83fP51T2KW6Pvr1S+2nfuD0XN7+YeTvnkZOvpfmqUmLiMMYcMcb0A54BDtivZ4wxfY0xR8pzEBFpB/QBfgIijTHH7EXHgUj7fUvgsNNmCfa8kuaff4wpIrJORNYlJdW+O0bDA8NpF9xOR8pV9d65vHPM3jqbi5pdREzTmErvb0LXCSRmJrLi0IoqiE6Ba2NVrTDGvGa/vinvAUSkETAf+KPduO68b4ML7SWuMMbMMMbEGWPiIiIiqmKX1c4R6WD9ifXk5ed5OhSlPOaLvV+QmJlY6dJGgStaXUHLRi35aLs2klcVl8aqqij7fo/5wL+NMQvs2SfsKijsnwV3oR8BWjtt3sqeV9L8OscR6eB0zmn2nNrj6VCU8ojc/FxmbZlFj7Ae9G/ev0r26e3lzbgu41ifuJ6dJ3dWyT7rO7clDhERYBaw3RjzktOiRUBBz6iJwEKn+bfYvasuBtLsKq3/AkNFpIndKD7UnlfnFDzYSds5VH21/OByDp8+zB3Rd2B9hFSN6zpdR4B3AHN3zK2yfdZn7ixxXArcDFwpIhvs1wjgBWCIiOwGBtvTAEuAfcAe4F3g9wDGmJPAs8DP9usZe16d06JRC5o1bKaJQ9VLxhje3fwu7UPaM7DNwCrdd4h/CCPbj+TLfV+Slp1Wpfuuj3zctWNjzBqgpK8Mg4pZ32D15CpuX//CGmyxznNEOvjx6I8YY6r0G5dSNd23Cd+yO3U3z1/2PF5S9d9pJ3SbwPzd81mwe0G570RXRbm1jUOVnyPSQUpWCodOH/J0KEpVG2MMMzbPoGWjlgyPck9v+85NOhMXGce8nfO0A0olaeKoYRxNtZ1D1T/rTqxjU9ImJvWYhK+X+8ZQndBtAkcyjrA6YbXbjlEfaOKoYaJComji30QTRw2RfDaZ1YdX8/ovr3PfivtYun+pp0Oqk97d9C5hAWGM7jS67JUrYWDrgTRr2EzHr6okt7VxqIoRERyRDk0cHpCZk8m2lG1sSd7C5uTNbEnewtEzRwFrtNUm/k1YdXgVXuLFsHbDPBxt3bEleQs/HPuBBxwP4O/t79Zj+Xj5MLbLWF5Z/wp7T+2lQ+MObj1eXaWJowa6qNlFfH3oax5c9SD3xNxD+8btPR1SnZOTn8Oe1D2FCWJz8mb2pe0j3+QD0LJRS3pF9GJCtwlEh0fTNbQrIsKdy+9k2nfTCPIN4pKWl3j4LOqGmZtnEuQXxA1dbqiW443pNIa3NrzF3B1zefzix6vlmHWNJo4a6PrO15OSlcKH2z7k60NfMzJqJHf3vpvWwa3L3lhdwBhDwukENidvLkwU209uJzsvG4DG/o3pGd6TIW2H0DO8Jz3DexIaEFrsvl4f9Dq3fXUbf1z1R2YMmVElQ2LUZ3tP7eWbQ99wV++7aOjbsFqO2SSgCVdFXcWivYv4Q+wfCPILqpbj1iVSzMjptV5cXJxZt26dp8OotNSsVN7b8h5zd8wlJz+H0R1Hc2evO2neqLmnQ6vRUs6msDVla5FEUdB3P8A7gO5h3ekR3oPo8Gh6hvekVaNW5er6nHw2mYlLJ5Kancp7w96r1Oit9d2j3z3K14e+ZtmYZTQOaFxtx92Wso2xi8fy8EUPc3P3m6vtuDWdiMQbY+LKXE8TR82XlJnEzM0z+XTXp4BVIrk9+nYiGtTOMbmq0vntEltTtnIkwxqRxku86Ni4Y2GCiA6PpkPjDvh4Vb6gfSTjCLcsvYW8/Dzev+p92gS3qfQ+65uE0wlc/Z+rubHbjTx00UPVfvybl9zMyayTfHHdF265b6Q20sRRhxJHgWMZx3hn0zss3LMQHy8fxnUdx209b6NJQJ17PEmxcvNz2XOqaLvE3lN7i7RLFCSInuE96RbajQa+DdwWz75T+5j41UQa+jZkzvA5RDaMLHsjVegvP/6FBbsXsPS3Sz1y7ZbuX8rD3z7MG4Pe4IpWV1T78WsiTRx1MHEUOJx+mLc2vsXifYsJ9Ankpu43MbHHRIL9gj0dWpXbd2ofS/YvYe3xtWxP2U5WXhZgDSFRkCSiw6PpEdaDsMCwao9va/JWbvvvbTRv2JzZw2dXa3VLbZaUmcTw+cO5tuO1PNn/SY/EkJOXw7D5w+gc2pm3B7/tkRhqGk0cdThxFNh7ai9vbniTZQeXEeQXxK09buXGbjdWWyOjuxw/c5yl+5eyZP8SdpzcgZd4ER0eTa+IXlZpIqwnrYLK1y7hTmuPreXur++mS2gXZg6d6dZSTl3x0rqXmLNtDotHL/Zop4+3NrzFmxvf5IvRX9AupJ3H4qgpNHHUg8RRYMfJHbyx4Q1WHV5FE/8mTI6ezA1dbiDQJ9DTobksLTuNZQeXsWTfEuJPxGMw9ArvxYj2IxjWbhjhgeGeDrFUKw6t4IFVDxDXLI43B72Jn7efp0OqsdKy0xj62VAGtB7A9CumezSW5LPJDPlsCGO7jGVa32llb1DHaeKoR4mjwOakzby+4XW+P/o94YHh3BF9B7/r/Lsa+yGWmZPJ6oTVLNm3hDVH15Cbn0tUSBQjo0YyImpEret+vGjvIh5b8xiD2wzm77/5e5U0wtdFb218izc3vMn8a+fTuUlnT4fDI98+wrcJ3/L19V/X+tJ6ZWniqIeJo8C64+t47ZfXWJ+4nmYNm3FXr7u4tuO1bh0DyFU5+Tn8ePRHvtz/JSsOreBs7lmaNmjKiKgRjIgaUXijXW314bYPmf7zdEZ3HM3TlzytvXXOk5mTydD5Q+nTtA+vXfmap8MBYGPSRm5achOP9XuMcV3HeTocj3I1cehXojoorlkcs4fP5odjP/D6L6/z1A9PMWvLLO7ufTcjokbg7eVdrfHkm3w2Jm3ky31fsuzAMlKzUwn2C2Zke6tk4Yh01JkP2Ju630T6uXTe2vgWQX5BPBT3UK1OhFXt012fkpadVmWPha0KvcJ70SOsB3N3zGVsl7H6+3KBJo46SkS4pMUl9G/en9UJ1iB9j655lJmbZ3JPzD0MbjvY7R/Wu1J3sWTfEpbuX8rRM0cJ8A5gQOsBjGw/kktbXIqvt+dLQO5wd++7SctO44NtH9DYvzFTek3xdEg1wrm8c8zZOod+zfrRO6K3p8MpJCJM6DaBx9Y8xo/HfqR/i6p5ZG1dpomjjhMRBrQewBWtrmD5weW8ueFNHlz9IF1DuzI1ZipXtLqiSr9hHck4UtgjanfqbrzFm/4t+jO1z1SubHNlvahDFhEe6fsI6efSee2X1wj2C673VSAAC/cuJOlsEs9f/rynQ7nA8HbDeXHdi3y04yNNHC7QxFFPFIzoOrjNYJbsX8JbG99i6oqp9ArvxT197qF/8/4VTiAns06y7MAyluxfwi+JvwAQExHDo/0eZVi7YSWO+1SXeYkXz1z6DBnnMnj+p+cJ8gtiZPuRng7LY3Lzc/nX5n8RHR5Nv2b9PB3OBfy8/RjTaQwzN88k4XQCrYJaeTqkGk0bx+upnPwcFu1ZxNub3ub4meM4Ih3c2+deHJEOl7Y/k3OGFYdWsGT/En44+gN5Jo+OjTsysv1Ihrcbrv94tuy8bO7++m5+OfELr1z5Sr29Q/nLfV8y7btpvDLwFa5sc6WnwynW8TPHGT5/ODd3v5kH4x70dDgeob2qNHG45FzeOT7b9Rnvbn6X5LPJXNLiEqbGTCU6IvqCdXPycvjf0f+xZN8SVh5eSVZeFs0bNrd6RLUfUSO6VtZEGecymLxsMntP7eWdIe+4nJzrinyTz5hFYzDGsGDUghrdEeLBVQ/y47Ef+fr6r2vVfVBVRROHJo5yOZt7lnk75jFryyxOZZ9iQOsBTI2ZSqcmnYg/Ec+S/UtYdmAZ6efSaezfmGHthjEiagQxTWNq9AdBTXEy6yS3fnUrSZlJzBo2i+5h3T0dUrVZeWgl9628j+cve55rOlzj6XBKFX8inlu/upUn+z/J7zr/ztPhVDtNHJo4KuRMzhk+3PYhc7bO4XTOaUIDQjmZdZJAn0AGth7IyPYj6d+if424J6S2OX7mOLcsvYXsvGxmD59NVEiUp0NyO2MMNy65kZNZJ1l83eIaf1OkMYbrv7iefPKZf838etc119XEoV8VVRENfRtyZ+87WTpmKXf2upO4yDimXz6dVTesYvoV07mi1RWaNCqoWcNmzBgyA4Apy6dw/MxxD0fkfmuPr2Vz8mZu63lbjU8a8GvX3N2pu1l3Qr98lkRLHEpVs+0p27ntv7cRHhjOnKvm1OleZ7cvu529p/by1Ziv3P488aqSlZvF4M8G07dZX14a8JKnw6lWWuJQqobqFtaN1we9zrEzx7hr+V1knMvwdEhusSlpEz8d+4mJ3SfWmqQBEOATwG87/ZYVh1bUulJhdl524SOR3cltiUNE/iUiiSKyxWleqIgsF5Hd9s8m9nwRkVdFZI+IbBKRWKdtJtrr7xaRie6KV6nq5Ih08NKAl9idupt7V9xLVm6Wp0OqcjM3zyTYL5jru1zv6VDKbVyXcRgM83bO83QoLjuacZSJSyfy15/+6vZjubPEMRsYft68acA3xphOwDf2NMBVQCf7NQV4C6xEAzwJ9AP6Ak8WJBularsrWl3Bc5c9R/yJeB5a/RA5+TmeDqnK7E7dzcrDK2vt82FaNGrBgFYDmL9rfrV8g6+s749+z9jFYzmYfpDLW13u9uO5LXEYY74FTp43exQwx34/BxjtNP99Y/kRaCwizYFhwHJjzEljTCqwnAuTkVK11oj2I3is32OsSljFE/97ovAxuLXdrC2zCPQJZELXCZ4OpcImdJtAanYqX+3/ytOhlCjf5PPupne5a/ldhAeGM3fkXAa1GeT241Z3G0ekMeaY/f44UPCg4ZbAYaf1Eux5Jc2/gIhMEZF1IrIuKSmpaqNWyo3Gdh3LfX3uY/G+xbyw9gVqe4eVw6cPs3T/Um7ofEOtfpRu32Z96RDSgY92fFQjfyfp59L5w8o/8OovrzK83XD+PeLf1fYUQ481jhvrN1Flvw1jzAxjTJwxJi4iIqKqdqtUtbg9+nYmdp/I3B1zeXPjm54Op1Le2/Ie3uLNLT1u8XQolSIijO86nm0p29iYtNHT4RSxK3UX4xePZ03CGqb1ncb0K6ZX6yOLqztxnLCroLB/JtrzjwDOj3trZc8rab5SdYqI8GDcg1zX8Tre3vg2H2z7wNMhVUhiZiKf7/mc0R1H07RBU0+HU2nXdLiGIN8gPtrxkadDKfTlvi+5aclNZOZmMmvYLG7sdmO136hY3YljEVDQM2oisNBp/i1276qLgTS7Suu/wFARaWI3ig+15ylV54gIT/R/gsFtBvO3n//Gwj0Ly96ohnl/6/vkmTwm9Zzk6VCqRAPfBozqOIrlB5aTlOnZKvCcvBz++tNfmfbdNLqFduOTqz8hNjK27A3dwJ3dcecCPwBdRCRBRCYDLwBDRGQ3MNieBlgC7AP2AO8CvwcwxpwEngV+tl/P2POUqpN8vHyYfsV0+jXvx5PfP8k3h77xdEguO5V1ik92fcJVUVfROqh2PS++NOO7jifP5PHprk89FkNiZiK3/fc2PtrxETd3v5mZw2YS0cBzVfJ65/j5Dv8MreKgno1Ro2qWzJxM7lh2B9tPbuetwW/Rr3nNe4bF+d7c8CZvbXyLBdcuoFOTTp4Op0r9/uvfs/3kdpaNWVbtT65cd3wdf1r9JzJzM3nmkmcYHuW+jqV653hF7FsNswbDpxPhTLKno1H1WAPfBrwx6A3aBrflvhX3sSV5S9kbedCZnDP8e/u/Gdh6YJ1LGmB1zU0+m8yyg8uq7ZjGGOZsncPty24nyC+Ij0Z85NakUR6aOJy1vRQGPQk7l8Ib/WD7F56OSNVjjQMa886Qd2gS0IS7vr6Lvaf2ejqkEn2681PSz6Vze/Ttng7FLS5pcQltg9tWWyN5Zk4mD337EP9Y9w8GtB7A3JFz6dikY7Uc2xWaOJx5+8DlD8CU1RDcAubdBPPvgExtVlGe0bRBU94d8i6+Xr5MWTaFIxk1r1Nhdl42c7bNoV/zfvSK6OXpcNzCS7wY33U8m5I2sTV5q1uPtS9tH+O/HM/yg8u533E/Lw94mUZ+jdx6zPLSxFGcyO5wxwoY8ChsXQBv9odd2plLeUbr4Na8M+QdzuadZcqyKSSfrVnVqAv3LCT5bDJ3RN/h6VDcalSHUTTwaeDWUsfyg8sZv3g8qVmpzBgyg9t63lYjnwmiiaMk3r4w4BErgTQIg49ugM/vgaw0T0em6qHOTTrz5qA3STqbxJ3L72TloZXsSt1FZk6mR+PKzc/lX1v+Ra/wXvRt1tejsbhbI79GXNvhWpbuX0rK2ZQq3Xdufi4vrXuJB1Y9QIfGHfjkmk9qdIcI7VXlitxsWD0d1rwMQc3h2tego/vHg1HqfN8f+Z77Vt5XZOC90IBQWjZqSatGrWgZ1NJ6H9SKlo1a0qxhM7c+eOuLvV/w6JpHeXXgqwxsM9Btx6kp9qXtY9Tno7i3z71M6TWlSvaZcjaFh799mLXH1zK2y1gevuhh/Lz9qmTf5aWPjnXHg5wS4uHzuyB5FzgmwdBnwT+o6o+jVCnSz6VzMO0gRzKOkJCRQMLpBOv96QSOnzlOrsktXNdbvGnWsFmRZOL8PiwgrMJVIfkmn98u/C0iwvxr59ebZ89PWTaFvWnWw6kqm5Q3Jm3kgVUPkJadxp8v/jOjOo6qoigrxtXEUfOf5ViTtHLAnd/Cyufg+9dh7zcw6k2Icv8wxkoVCPYLJjoimuiI6AuW5ebnkpiZWJhMDp8+zJGMIxzJOMLqw6tJySpaxRLoE1iYTIpLLqUNib7y8Er2pu3lhctfqDdJA6yuufeuuJcVh1YwrN2wCu3DGMMnOz/hhZ9fILJBJB+O+JCuoV2rOFL30RJHRR36ET6/G07ug753wuCnwK/6BhlTqiLO5p7laMZREk4nkJDxa0ml4GdmbtE2kyb+TYotqbRq1IqHv32YU9mn+OK6L2rF88SrSl5+HiP/M5JmDZsxe/jscm9/Nvcsf/nxLyzau4jLW17OXy//KyH+IVUfaAVoicPd2lwMd62Br5+Gte/AnuUw+m1oU3MbtJQK9AmkQ+MOdGjc4YJlxhhOZZ8qtgpsa8pWvj74dZFqMIAn+j9Rr5IGgLeXN+O7jucf6/7BzpM76RLaxeVtD6cf5v5V97MrdRe/7/177ux9Z60srWmJoyrs/xYW3gOnDsMlU2Hg4+AbUH3HV6oa5OXncSLzRJHSyQ1dbnBr43tNlZadxpDPhjAiagRPXfKUS9t8m/At076bhiC8cPkL1fKkvvLSxvHqTBwA2adh2eMQPxvCO1ulj1aO6o1BKVVtnv7haRbvXczX139dalVTXn4eb296m7c3vk230G68NOAlWgW1qsZIXadjVVU3/yC45hW4aQGcOwOzhsA3z1hdeZVSdc74ruPJystiwe4FJa6Tlp3GPSvu4e2NbzOqwyjev+r9Gps0ykMTR1XrOAju/h56j4PvXoQZA+HYJk9HpZSqYp2bdCYuMo55O+eRl593wfJtKdsYu3gsa4+t5c8X/5lnL32WAJ+6UYWticMdAhvD6Ddh/DzITIZ3B8Kq6ZCX4+nIlFJVaEK3CVZX54TVReb/Z/d/uHnJzeSZPOYMn8MNXW6okUOHVJQmDnfqMhx+/yP0uA5WPQ8zB8GJbZ6OSilVRQa2Hkizhs0Kx686l3eOp394mie+f4I+kX2Yd/W8Yu+3qe00cbhbg1AYMxNueB/SjsCM38B3L0FebtnbKqVqNB8vH8Z2GctPx35izZE1TFw6kc92fcbknpN5e/DbhAaEejpEt9BeVdXpTDIsvh+2L4KWcXDd2xBeQx56k5sNKXsgcTsk7YQk++eZZKvh3z8YAoJ/fe8fdN50cctDrJ/V/MQ0papTalYqgz8dzLn8czTybcRfLvsLg9rUzrHstDtuTUwcAMbAlvmw5E+QcxYGPQH97gIv7+o5fk6WlSCSdlivgkRxch8Yu4FPvCC0PUR0hUaRVi+x7HSry3FWmvUzOx2y0iHfhXYbn8BiEo1TYrkgEYVcuL63H5j8Yl7GirvYZfby/DKWl7Z9/nnTShXjpUNLWJe+j+c7jKVdoOeeBQ5AUDNoGVuhTTVx1NTEUeD0cfjij7BrKbTpbzWmh7avuv3nZFmDMSbt/DVJJO2wE4T9ASje1jGbdrWSRMErvBP4+Lt+nMJE4pRUsk9biSX7NGSnnTd9/vJ0oO79HSrlET1+C9e/V6FNNXHU9MQB1rfdjXNh6TTrm/uQZyBuMniVo+kp5+yvCcK5min1QNEEEdbh18RQkCjCOrqeINwpPx9yzpyXWNKLTuflWCWhYl9ildhKXO7Cy8uV9QSoOz1jVB0V2BiatKvQppo4akPiKJB2BBbdaxZXtcYAAAf7SURBVI22G3UFXPs6NGlbdJ1zmU4liO2/JorUAxR+W/fygdAOF5YgwjqCj2fG91dK1R6aOGpT4gCr9LF+Dvz3MWv60j/CuQynEsRBiiSIsI4XliBCO2iCUEpVmI6OWwFJp7P5eO0hIoL8aRrsT9OgACKC/Alr6IePt5t7LouA41ZoP9AaMHHlX8DL10oQLfpA7/FOJYgO2lNJKeUxmjicHDqZyYvLd10wXwTCGvoRYSeSpvbLeu80L9ifBn6VvKRN2sItiyD9iNU7QhOEUqqGqTWJQ0SGA68A3sBMY8wLVX0MR9sm7PzLcJIzzpGYnkXi6WySTmcX/kw6bc3bfeI0Saezyc2/sJqvoZ83TYMDiGjkT0Rw8QkmIsif0AZ+eHmV0NDq5QWNW1f16SmlVJWoFYlDRLyBN4AhQALws4gsMsZU+fgd/j7etGwcSMvGgaWul59vOHU2h8TTWSSmF00wiXaC2X40ndWns8nIvvAucR8vIbyRf5Fk0jTInwg76YQ18sPKK4KI1ZdHROyfIPZ87Gl7TWuZ03IpXO7afgrG0ylY7iVS+NN6Wet4Oc37dfmv69elcXmUUkXVisQB9AX2GGP2AYjIx8AowGMDP3l5CaEN/Qht6EfXZqWvm3kut2hiSc8iKSObxHRr3rG0LDYmpJFyJpu60lfBOZmUnWicl9vrexVdv6akoZqSEGtGFKomGtAlgsdGdnfrMWpL4mgJHHaaTgCKPKNVRKYAUwDatGlTfZG5oIGfD23DfGgb1rDU9XLz8jl55hyJp7NJzTxHvrEe52kA/r+9u4uxawrDOP5/TInSqEZD6JRpoiG+KyI+EhflglS4cIHgQtwQrRJBuXYlIhSR1FckGr2oEpGGSolICKVoVUmkqh/aaC9aKqLaeVzsPc6ZaulmTteZOc8vmcze6+zZec+aOfOetfY+6zUYVx90dnV/1dBjrWTjtsfajq+PZd/2fc4zdNNW+88NeuhYM+hqf/Cv/VabXY3CWvsHcXxbm20GB/d//N5uyaZdE0aXBBJd6YRjOr90+2hJHP/K9gJgAVS34xYO5z8Z13cYxx9zJMcfgl98RMR/NVpWx90MtF8t7q/bIiLiEBstiWMFMF3SNElHADcAbxSOKSKiJ42KqSrbeyTNBt6muh33BdtrCocVEdGTRkXiALC9FFhaOo6IiF43WqaqIiKiSyRxREREI0kcERHRSBJHREQ0MibrcUjaBvzwP04xGdg+QuGMdumL4dIfLemL4cZCf5xi+1+Lpo/JxPF/Sfr0YIqZ9IL0xXDpj5b0xXC91B+ZqoqIiEaSOCIiopEkjv1bUDqALpK+GC790ZK+GK5n+iPXOCIiopGMOCIiopEkjoiIaCSJo42kKyV9K+k7SfNKx1OSpKmS3pP0taQ1kuaWjqk0SX2SPpf0ZulYSpN0rKTFkr6RtFbSxaVjKknSPfXr5CtJr0ga09XYkjhqkvqAp4GrgDOAGyV1tnBvd9sD3Gv7DOAi4M4e7w+AucDa0kF0iSeAt2yfDpxLD/eLpCnAXcAFts+iKv1wQ9moOiuJo+VC4Dvb62zvBhYB1xaOqRjbW2yvrLd/ofrHMKVsVOVI6gdmAc+VjqU0SROBy4DnAWzvtr2jbFTFjQPGSxoHHAX8WDiejkriaJkCbGzb30QP/6NsJ2kAmAF8XDaSoh4H7gcGSwfSBaYB24AX66m75yQdXTqoUmxvBh4FNgBbgJ22l5WNqrOSOOIfSZoAvArcbfvn0vGUIOlq4Cfbn5WOpUuMA84HnrE9A/gV6NlrgpImUc1OTANOAo6WdHPZqDoriaNlMzC1bb+/butZkg6nShoLbS8pHU9BlwLXSFpPNYU5U9LLZUMqahOwyfbQCHQxVSLpVVcA39veZvsPYAlwSeGYOiqJo2UFMF3SNElHUF3ceqNwTMVIEtUc9lrbj5WOpyTbD9rutz1A9Xfxru0x/Y7yn9jeCmyUdFrddDnwdcGQStsAXCTpqPp1czlj/GaBUVNzvNNs75E0G3ib6q6IF2yvKRxWSZcCtwCrJX1Rtz1U136PmAMsrN9krQNuLRxPMbY/lrQYWEl1N+LnjPHlR7LkSERENJKpqoiIaCSJIyIiGkniiIiIRpI4IiKikSSOiIhoJIkjogFJeyV90fY1Yp+YljQg6auROl9Ep+RzHBHN/Gb7vNJBRJSUEUfECJC0XtIjklZL+kTSqXX7gKR3Ja2StFzSyXX7CZJek/Rl/TW0REWfpGfr2g7LJI2vj7+rro2yStKiQk8zAkjiiGhq/D5TVde3PbbT9tnAU1Sr6QI8Cbxk+xxgITC/bp8PvG/7XKp1noZWKZgOPG37TGAHcF3dPg+YUZ/n9k49uYiDkU+ORzQgaZftCftpXw/MtL2uXhxyq+3jJG0HTrT9R92+xfZkSduAftu/t51jAHjH9vR6/wHgcNsPS3oL2AW8Drxue1eHn2rEAWXEETFyfIDtJn5v295L6zrkLKoKlecDK+qCQRFFJHFEjJzr275/VG9/SKuM6E3AB/X2cuAO+KuW+cQDnVTSYcBU2+8BDwATgb+NeiIOlbxriWhmfNtqwVDV3R66JXeSpFVUo4Yb67Y5VJXy7qOqmje0iuxcYIGk26hGFndQVY/bnz7g5Tq5CJifUq1RUq5xRIyA+hrHBba3l44lotMyVRUREY1kxBEREY1kxBEREY0kcURERCNJHBER0UgSR0RENJLEERERjfwJyZikfbF+mN4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Listing 3.5 –Train model with various optimizers\n",
    "\n",
    "\n",
    "#Define loss function\n",
    "loss_function = nn.BCELoss()  #Binary Crosss Entropy Loss\n",
    "num_epochs = 10\n",
    "batch_size=16\n",
    "\n",
    "#Define a model object from the class defined earlier\n",
    "model = NeuralNetwork()\n",
    "\n",
    "#Train network using RMSProp optimizer\n",
    "rmsprp_optimizer = tch.optim.RMSprop(model.parameters()\n",
    ", lr=0.01, alpha=0.9\n",
    ", eps=1e-08, weight_decay=0.1\n",
    ", momentum=0.1, centered=True)\n",
    "print(\"RMSProp...\")\n",
    "rmsprop_loss = train_network(model,rmsprp_optimizer,loss_function\n",
    ",num_epochs,batch_size,X_train,Y_train)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#Train network using Adam optimizer\n",
    "\n",
    "model = NeuralNetwork()\n",
    "adam_optimizer = tch.optim.Adam(model.parameters(),lr= 0.001)\n",
    "print(\"Adam...\")\n",
    "adam_loss = train_network(model,adam_optimizer,loss_function\n",
    ",num_epochs,batch_size,X_train,Y_train)\n",
    "\n",
    "#Train network using SGD optimizer\n",
    "\n",
    "model = NeuralNetwork()\n",
    "sgd_optimizer = tch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\n",
    "print(\"SGD...\")\n",
    "sgd_loss = train_network(model,sgd_optimizer,loss_function\n",
    ",num_epochs,batch_size,X_train,Y_train) \n",
    "\n",
    "#Plot the losses for each optimizer across epochs\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "epochs = range(0,10)\n",
    "\n",
    "ax = plt.subplot(111)\n",
    "ax.plot(adam_loss,label=\"ADAM\")\n",
    "ax.plot(sgd_loss,label=\"SGD\")\n",
    "ax.plot(rmsprop_loss,label=\"RMSProp\")\n",
    "ax.legend()\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Overall Loss\")\n",
    "plt.title(\"Loss across epochs for different optimizers\")\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
